From 3905d888b656268bf9b9e1932fc45415554d9218 Mon Sep 17 00:00:00 2001 From: "djm@kirby.fc.hp.com" Date: Wed, 30 Jun 2004 22:26:29 +0000 Subject: [PATCH] bitkeeper revision 1.1041.2.3 (40e33e15ASwSHg8_daLqUfOJezgo6A) Encapsulate shadow_mode for minimal perturbation portability and move shadow.[ch] to machdep layer --- .rootkeys | 5 +- BitKeeper/etc/logging_ok | 1 + xen/{common => arch/x86}/shadow.c | 0 xen/include/asm-x86/shadow.h | 604 ++++++++++++++++++++++++++++++ xen/include/xen/shadow.h | 600 +---------------------------- 5 files changed, 609 insertions(+), 601 deletions(-) rename xen/{common => arch/x86}/shadow.c (100%) create mode 100644 xen/include/asm-x86/shadow.h diff --git a/.rootkeys b/.rootkeys index feb8ec1995..26357f0617 100644 --- a/.rootkeys +++ b/.rootkeys @@ -326,6 +326,7 @@ 3ddb79bc1_2bAt67x9MFCP4AZrQnvQ xen/arch/x86/process.c 3ddb79bc7KxGCEJsgBnkDX7XjD_ZEQ xen/arch/x86/rwlock.c 3ddb79bcrD6Z_rUvSDgrvjyb4846Eg xen/arch/x86/setup.c +405b8599xI_PoEr3zZoJ2on-jdn7iw xen/arch/x86/shadow.c 3ddb79bcSx2e8JSR3pdSGa8x1ScYzA xen/arch/x86/smp.c 3ddb79bcfUN3-UBCPzX26IU8bq-3aw xen/arch/x86/smpboot.c 3ddb79bc-Udq7ol-NX4q9XsYnN7A2Q xen/arch/x86/time.c @@ -353,7 +354,6 @@ 40589968dD2D1aejwSOvrROg7fOvGQ xen/common/sched_bvt.c 40589968be_t_n0-w6ggceW7h-sx0w xen/common/sched_rrobin.c 3e397e6619PgAfBbw2XFbXkewvUWgw xen/common/schedule.c -405b8599xI_PoEr3zZoJ2on-jdn7iw xen/common/shadow.c 3ddb79bdB9RNMnkQnUyZ5C9hhMSQQw xen/common/slab.c 3ddb79bd0gVQYmL2zvuJnldvD0AGxQ xen/common/softirq.c 3e7f358awXBC3Vw-wFRwPw18qL1khg xen/common/string.c @@ -443,6 +443,7 @@ 3ddb79c2QF5-pZGzuX4QukPCDAl59A xen/include/asm-x86/processor.h 40cf1596bim9F9DNdV75klgRSZ6Y2A xen/include/asm-x86/ptrace.h 3ddb79c2plf7ciNgoNjU-RsbUzawsw xen/include/asm-x86/rwlock.h +405b8599BsDsDwKEJLS0XipaiQW3TA xen/include/asm-x86/shadow.h 3ddb79c3Hgbb2g8CyWLMCK-6_ZVQSQ xen/include/asm-x86/smp.h 3ddb79c3jn8ALV_S9W5aeTYUQRKBpg xen/include/asm-x86/smpboot.h 3ddb79c3NiyQE2vQnyGiaBnNjBO1rA xen/include/asm-x86/spinlock.h @@ -500,7 +501,7 @@ 40589969nPq3DMzv24RDb5LXE9brHw xen/include/xen/sched-if.h 3ddb79c0LzqqS0LhAQ50ekgj4oGl7Q xen/include/xen/sched.h 403a06a7H0hpHcKpAiDe5BPnaXWTlA xen/include/xen/serial.h -405b8599BsDsDwKEJLS0XipaiQW3TA xen/include/xen/shadow.h +40e3392dib7GrcBAu5cT-EUZTYzeEQ xen/include/xen/shadow.h 3ddb79c14dXIhP7C2ahnoD08K90G_w xen/include/xen/slab.h 3ddb79c09xbS-xxfKxuV3JETIhBzmg xen/include/xen/smp.h 3ddb79c1Vi5VleJAOKHAlY0G2zAsgw xen/include/xen/softirq.h diff --git a/BitKeeper/etc/logging_ok b/BitKeeper/etc/logging_ok index 04eb20bb84..d2431905b5 100644 --- a/BitKeeper/etc/logging_ok +++ b/BitKeeper/etc/logging_ok @@ -9,6 +9,7 @@ bd240@labyrinth.cl.cam.ac.uk br260@br260.wolfson.cam.ac.uk br260@labyrinth.cl.cam.ac.uk br260@laudney.cl.cam.ac.uk +djm@kirby.fc.hp.com gm281@boulderdash.cl.cam.ac.uk iap10@freefall.cl.cam.ac.uk iap10@labyrinth.cl.cam.ac.uk diff --git a/xen/common/shadow.c b/xen/arch/x86/shadow.c similarity index 100% rename from xen/common/shadow.c rename to xen/arch/x86/shadow.c diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h new file mode 100644 index 0000000000..df24329b2b --- /dev/null +++ b/xen/include/asm-x86/shadow.h @@ -0,0 +1,604 @@ +/* -*- Mode:C; c-basic-offset:4; tab-width:4 -*- */ + +#ifndef _XEN_SHADOW_H +#define _XEN_SHADOW_H + +#include +#include +#include +#include + + +/* Shadow PT flag bits in pfn_info */ +#define PSH_shadowed (1<<31) /* page has a shadow. PFN points to shadow */ +#define PSH_pending (1<<29) /* page is in the process of being shadowed */ +#define PSH_pfn_mask ((1<<21)-1) + +/* Shadow PT operation mode : shadowmode variable in mm_struct */ +#define SHM_test (1) /* just run domain on shadow PTs */ +#define SHM_logdirty (2) /* log pages that are dirtied */ +#define SHM_translate (3) /* lookup machine pages in translation table */ +//#define SHM_cow (4) /* copy on write all dirtied pages */ + + +#define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START) +#define shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START+(SH_LINEAR_PT_VIRT_START>>(L2_PAGETABLE_SHIFT-L1_PAGETABLE_SHIFT)))) + +extern void shadow_mode_init(void); +extern int shadow_mode_control( struct domain *p, dom0_shadow_control_t *sc ); +extern int shadow_fault( unsigned long va, long error_code ); +extern void shadow_l1_normal_pt_update( unsigned long pa, unsigned long gpte, + unsigned long *prev_spfn_ptr, + l1_pgentry_t **prev_spl1e_ptr ); +extern void shadow_l2_normal_pt_update( unsigned long pa, unsigned long gpte ); +extern void unshadow_table( unsigned long gpfn, unsigned int type ); +extern int shadow_mode_enable( struct domain *p, unsigned int mode ); +extern void shadow_mode_disable( struct domain *p ); +extern unsigned long shadow_l2_table( + struct mm_struct *m, unsigned long gpfn ); + +#define SHADOW_DEBUG 0 +#define SHADOW_HASH_DEBUG 0 +#define SHADOW_OPTIMISE 1 + +struct shadow_status { + unsigned long pfn; // gpfn + unsigned long spfn_and_flags; // spfn plus flags + struct shadow_status *next; // use pull-to-front list. +}; + +#define shadow_ht_extra_size 128 /*128*/ +#define shadow_ht_buckets 256 /*256*/ + +#ifndef NDEBUG +#define SH_LOG(_f, _a...) \ +printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \ + current->domain , __LINE__ , ## _a ) +#else +#define SH_LOG(_f, _a...) +#endif + +#if SHADOW_DEBUG +#define SH_VLOG(_f, _a...) \ + printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \ + current->domain , __LINE__ , ## _a ) +#else +#define SH_VLOG(_f, _a...) +#endif + +#if 0 +#define SH_VVLOG(_f, _a...) \ + printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \ + current->domain , __LINE__ , ## _a ) +#else +#define SH_VVLOG(_f, _a...) +#endif + + +/************************************************************************/ + +#define shadow_mode(d) (d->mm.shadow_mode) +#define shadow_lock_init(d) spin_lock_init(&d->mm.shadow_lock) + +/************************************************************************/ + +static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn ) +{ + unsigned int pfn; + int rc = 0; + + ASSERT(spin_is_locked(&m->shadow_lock)); + + pfn = machine_to_phys_mapping[mfn]; + + /* We use values with the top bit set to mark MFNs that aren't + really part of the domain's psuedo-physical memory map e.g. + the shared info frame. Nothing to do here... + */ + if ( unlikely(pfn & 0x80000000U) ) return rc; + + ASSERT(m->shadow_dirty_bitmap); + if( likely(pfnshadow_dirty_bitmap_size) ) + { + /* These updates occur with mm.shadow_lock held, so use + (__) version of test_and_set */ + if( __test_and_set_bit( pfn, m->shadow_dirty_bitmap ) == 0 ) + { + // if we set it + m->shadow_dirty_count++; + rc = 1; + } + } + else + { + extern void show_traceX(void); + SH_LOG("mark_dirty OOR! mfn=%x pfn=%x max=%x (mm %p)", + mfn, pfn, m->shadow_dirty_bitmap_size, m ); + SH_LOG("dom=%u caf=%08x taf=%08x\n", + frame_table[mfn].u.domain->domain, + frame_table[mfn].count_and_flags, + frame_table[mfn].type_and_flags ); + } + + return rc; +} + + +static inline int mark_dirty( struct mm_struct *m, unsigned int mfn ) +{ + int rc; + ASSERT(local_irq_is_enabled()); + //if(spin_is_locked(&m->shadow_lock)) printk("+"); + spin_lock(&m->shadow_lock); + rc = __mark_dirty( m, mfn ); + spin_unlock(&m->shadow_lock); + return rc; +} + + +/************************************************************************/ + +static inline void l1pte_write_fault( struct mm_struct *m, + unsigned long *gpte_p, unsigned long *spte_p ) +{ + unsigned long gpte = *gpte_p; + unsigned long spte = *spte_p; + + switch( m->shadow_mode ) + { + case SHM_test: + spte = gpte; + gpte |= _PAGE_DIRTY | _PAGE_ACCESSED; + spte |= _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED; + break; + + case SHM_logdirty: + spte = gpte; + gpte |= _PAGE_DIRTY | _PAGE_ACCESSED; + spte |= _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED; + __mark_dirty( m, (gpte >> PAGE_SHIFT) ); + break; + } + + *gpte_p = gpte; + *spte_p = spte; +} + +static inline void l1pte_read_fault( struct mm_struct *m, + unsigned long *gpte_p, unsigned long *spte_p ) +{ + unsigned long gpte = *gpte_p; + unsigned long spte = *spte_p; + + switch( m->shadow_mode ) + { + case SHM_test: + spte = gpte; + gpte |= _PAGE_ACCESSED; + spte |= _PAGE_ACCESSED; + if ( ! (gpte & _PAGE_DIRTY ) ) + spte &= ~ _PAGE_RW; + break; + + case SHM_logdirty: + spte = gpte; + gpte |= _PAGE_ACCESSED; + spte |= _PAGE_ACCESSED; + spte &= ~ _PAGE_RW; + break; + } + + *gpte_p = gpte; + *spte_p = spte; +} + +static inline void l1pte_no_fault( struct mm_struct *m, + unsigned long *gpte_p, unsigned long *spte_p ) +{ + unsigned long gpte = *gpte_p; + unsigned long spte = *spte_p; + + switch( m->shadow_mode ) + { + case SHM_test: + spte = 0; + if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) == + (_PAGE_PRESENT|_PAGE_ACCESSED) ) + { + spte = gpte; + if ( ! (gpte & _PAGE_DIRTY ) ) + spte &= ~ _PAGE_RW; + } + break; + + case SHM_logdirty: + spte = 0; + if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) == + (_PAGE_PRESENT|_PAGE_ACCESSED) ) + { + spte = gpte; + spte &= ~ _PAGE_RW; + } + + break; + } + + *gpte_p = gpte; + *spte_p = spte; +} + +static inline void l2pde_general( struct mm_struct *m, + unsigned long *gpde_p, unsigned long *spde_p, + unsigned long sl1pfn) +{ + unsigned long gpde = *gpde_p; + unsigned long spde = *spde_p; + + spde = 0; + + if ( sl1pfn ) + { + spde = (gpde & ~PAGE_MASK) | (sl1pfn<shadow_ht[j]; + if(a->pfn){live++; ASSERT(a->spfn_and_flags&PSH_pfn_mask);} + ASSERT((a->pfn&0xf0000000)==0); + ASSERT(a->pfn<0x00100000); + a=a->next; + while(a && live<9999) + { + live++; + if(a->pfn == 0 || a->spfn_and_flags == 0) + { + printk("XXX live=%d pfn=%08lx sp=%08lx next=%p\n", + live, a->pfn, a->spfn_and_flags, a->next); + BUG(); + } + ASSERT(a->pfn); + ASSERT((a->pfn&0xf0000000)==0); + ASSERT(a->pfn<0x00100000); + ASSERT(a->spfn_and_flags&PSH_pfn_mask); + a=a->next; + } + ASSERT(live<9999); + } + + a = m->shadow_ht_free; + while(a) { free++; a=a->next; } + + if(print) printk("Xlive=%d free=%d\n",live,free); + + abs=(perfc_value(shadow_l1_pages)+perfc_value(shadow_l2_pages))-live; + if( abs < -1 || abs > 1 ) + { + printk("live=%d free=%d l1=%d l2=%d\n",live,free, + perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages) ); + BUG(); + } + +} + +#else +#define shadow_audit(p, print) +#endif + + + +static inline struct shadow_status* hash_bucket( struct mm_struct *m, + unsigned int gpfn ) +{ + return &(m->shadow_ht[gpfn % shadow_ht_buckets]); +} + + +static inline unsigned long __shadow_status( struct mm_struct *m, + unsigned int gpfn ) +{ + struct shadow_status **ob, *b, *B = hash_bucket( m, gpfn ); + + b = B; + ob = NULL; + + SH_VVLOG("lookup gpfn=%08x bucket=%p", gpfn, b ); + shadow_audit(m,0); // if in debug mode + + do + { + if ( b->pfn == gpfn ) + { + unsigned long t; + struct shadow_status *x; + + // swap with head + t=B->pfn; B->pfn=b->pfn; b->pfn=t; + t=B->spfn_and_flags; B->spfn_and_flags=b->spfn_and_flags; + b->spfn_and_flags=t; + + if( ob ) + { // pull to front + *ob=b->next; + x=B->next; + B->next=b; + b->next=x; + } + return B->spfn_and_flags; + } +#if SHADOW_HASH_DEBUG + else + { + if(b!=B)ASSERT(b->pfn); + } +#endif + ob=&b->next; + b=b->next; + } + while (b); + + return 0; +} + +/* we can make this locking more fine grained e.g. per shadow page if it +ever becomes a problem, but since we need a spin lock on the hash table +anyway its probably not worth being too clever. */ + +static inline unsigned long get_shadow_status( struct mm_struct *m, + unsigned int gpfn ) +{ + unsigned long res; + + /* If we get here, we know that this domain is running in shadow mode. + We also know that some sort of update has happened to the underlying + page table page: either a PTE has been updated, or the page has + changed type. If we're in log dirty mode, we should set the approrpiate + bit in the dirty bitmap. + NB: the VA update path doesn't use this so needs to be handled + independnetly. + */ + + ASSERT(local_irq_is_enabled()); + //if(spin_is_locked(&m->shadow_lock)) printk("*"); + spin_lock(&m->shadow_lock); + + if( m->shadow_mode == SHM_logdirty ) + __mark_dirty( m, gpfn ); + + res = __shadow_status( m, gpfn ); + if (!res) spin_unlock(&m->shadow_lock); + return res; +} + + +static inline void put_shadow_status( struct mm_struct *m ) +{ + spin_unlock(&m->shadow_lock); +} + + +static inline void delete_shadow_status( struct mm_struct *m, + unsigned int gpfn ) +{ + struct shadow_status *b, *B, **ob; + + ASSERT(spin_is_locked(&m->shadow_lock)); + + B = b = hash_bucket( m, gpfn ); + + SH_VVLOG("delete gpfn=%08x bucket=%p", gpfn, b ); + shadow_audit(m,0); + ASSERT(gpfn); + + if( b->pfn == gpfn ) + { + if (b->next) + { + struct shadow_status *D=b->next; + b->spfn_and_flags = b->next->spfn_and_flags; + b->pfn = b->next->pfn; + + b->next = b->next->next; + D->next = m->shadow_ht_free; + D->pfn = 0; + D->spfn_and_flags = 0; + m->shadow_ht_free = D; + } + else + { + b->pfn = 0; + b->spfn_and_flags = 0; + } + +#if SHADOW_HASH_DEBUG + if( __shadow_status(m,gpfn) ) BUG(); + shadow_audit(m,0); +#endif + return; + } + + ob = &b->next; + b=b->next; + + do + { + if ( b->pfn == gpfn ) + { + b->pfn = 0; + b->spfn_and_flags = 0; + + // b is in the list + *ob=b->next; + b->next = m->shadow_ht_free; + m->shadow_ht_free = b; + +#if SHADOW_HASH_DEBUG + if( __shadow_status(m,gpfn) ) BUG(); +#endif + shadow_audit(m,0); + return; + } + + ob = &b->next; + b=b->next; + } + while (b); + + // if we got here, it wasn't in the list + BUG(); +} + + +static inline void set_shadow_status( struct mm_struct *m, + unsigned int gpfn, unsigned long s ) +{ + struct shadow_status *b, *B, *extra, **fptr; + int i; + + ASSERT(spin_is_locked(&m->shadow_lock)); + + B = b = hash_bucket( m, gpfn ); + + ASSERT(gpfn); + SH_VVLOG("set gpfn=%08x s=%08lx bucket=%p(%p)", gpfn, s, b, b->next ); + + shadow_audit(m,0); + + do + { + if ( b->pfn == gpfn ) + { + b->spfn_and_flags = s; + shadow_audit(m,0); + return; + } + + b=b->next; + } + while (b); + + // if we got here, this is an insert rather than update + + ASSERT( s ); // deletes must have succeeded by here + + if ( B->pfn == 0 ) + { + // we can use this head + ASSERT( B->next == 0 ); + B->pfn = gpfn; + B->spfn_and_flags = s; + shadow_audit(m,0); + return; + } + + if( unlikely(m->shadow_ht_free == NULL) ) + { + SH_LOG("allocate more shadow hashtable blocks"); + + // we need to allocate more space + extra = kmalloc(sizeof(void*) + (shadow_ht_extra_size * + sizeof(struct shadow_status))); + + if( ! extra ) BUG(); // should be more graceful here.... + + memset(extra, 0, sizeof(void*) + (shadow_ht_extra_size * + sizeof(struct shadow_status))); + + m->shadow_extras_count++; + + // add extras to free list + fptr = &m->shadow_ht_free; + for ( i=0; ishadow_ht_extras; + m->shadow_ht_extras = extra; + + } + + // should really put this in B to go right to front + b = m->shadow_ht_free; + m->shadow_ht_free = b->next; + b->spfn_and_flags = s; + b->pfn = gpfn; + b->next = B->next; + B->next = b; + + shadow_audit(m,0); + + return; +} + +static inline void __shadow_mk_pagetable( struct mm_struct *mm ) +{ + unsigned long gpfn, spfn=0; + + gpfn = pagetable_val(mm->pagetable) >> PAGE_SHIFT; + + if ( unlikely((spfn=__shadow_status(mm, gpfn)) == 0 ) ) + { + spfn = shadow_l2_table(mm, gpfn ); + } + mm->shadow_table = mk_pagetable(spfn<pagetable), mm->shadow_mode ); + + if ( unlikely(mm->shadow_mode) ) + { + ASSERT(local_irq_is_enabled()); + spin_lock(&mm->shadow_lock); + + __shadow_mk_pagetable( mm ); + + spin_unlock(&mm->shadow_lock); + } + + SH_VVLOG("leaving shadow_mk_pagetable( gptbase=%08lx, mode=%d ) sh=%08lx", + pagetable_val(mm->pagetable), mm->shadow_mode, + pagetable_val(mm->shadow_table) ); + +} + + +#if SHADOW_DEBUG +extern int check_pagetable(struct mm_struct *m, pagetable_t pt, char *s); +#else +#define check_pagetable(m, pt, s) ((void)0) +#endif + + +#endif /* XEN_SHADOW_H */ + + diff --git a/xen/include/xen/shadow.h b/xen/include/xen/shadow.h index e4dbd1b061..c59e02e14c 100644 --- a/xen/include/xen/shadow.h +++ b/xen/include/xen/shadow.h @@ -1,599 +1 @@ -/* -*- Mode:C; c-basic-offset:4; tab-width:4 -*- */ - -#ifndef _XEN_SHADOW_H -#define _XEN_SHADOW_H - -#include -#include -#include -#include - - -/* Shadow PT flag bits in pfn_info */ -#define PSH_shadowed (1<<31) /* page has a shadow. PFN points to shadow */ -#define PSH_pending (1<<29) /* page is in the process of being shadowed */ -#define PSH_pfn_mask ((1<<21)-1) - -/* Shadow PT operation mode : shadowmode variable in mm_struct */ -#define SHM_test (1) /* just run domain on shadow PTs */ -#define SHM_logdirty (2) /* log pages that are dirtied */ -#define SHM_translate (3) /* lookup machine pages in translation table */ -//#define SHM_cow (4) /* copy on write all dirtied pages */ - - -#define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START) -#define shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START+(SH_LINEAR_PT_VIRT_START>>(L2_PAGETABLE_SHIFT-L1_PAGETABLE_SHIFT)))) - -extern void shadow_mode_init(void); -extern int shadow_mode_control( struct domain *p, dom0_shadow_control_t *sc ); -extern int shadow_fault( unsigned long va, long error_code ); -extern void shadow_l1_normal_pt_update( unsigned long pa, unsigned long gpte, - unsigned long *prev_spfn_ptr, - l1_pgentry_t **prev_spl1e_ptr ); -extern void shadow_l2_normal_pt_update( unsigned long pa, unsigned long gpte ); -extern void unshadow_table( unsigned long gpfn, unsigned int type ); -extern int shadow_mode_enable( struct domain *p, unsigned int mode ); -extern void shadow_mode_disable( struct domain *p ); -extern unsigned long shadow_l2_table( - struct mm_struct *m, unsigned long gpfn ); - -#define SHADOW_DEBUG 0 -#define SHADOW_HASH_DEBUG 0 -#define SHADOW_OPTIMISE 1 - -struct shadow_status { - unsigned long pfn; // gpfn - unsigned long spfn_and_flags; // spfn plus flags - struct shadow_status *next; // use pull-to-front list. -}; - -#define shadow_ht_extra_size 128 /*128*/ -#define shadow_ht_buckets 256 /*256*/ - -#ifndef NDEBUG -#define SH_LOG(_f, _a...) \ -printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \ - current->domain , __LINE__ , ## _a ) -#else -#define SH_LOG(_f, _a...) -#endif - -#if SHADOW_DEBUG -#define SH_VLOG(_f, _a...) \ - printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \ - current->domain , __LINE__ , ## _a ) -#else -#define SH_VLOG(_f, _a...) -#endif - -#if 0 -#define SH_VVLOG(_f, _a...) \ - printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \ - current->domain , __LINE__ , ## _a ) -#else -#define SH_VVLOG(_f, _a...) -#endif - - -/************************************************************************/ - -static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn ) -{ - unsigned int pfn; - int rc = 0; - - ASSERT(spin_is_locked(&m->shadow_lock)); - - pfn = machine_to_phys_mapping[mfn]; - - /* We use values with the top bit set to mark MFNs that aren't - really part of the domain's psuedo-physical memory map e.g. - the shared info frame. Nothing to do here... - */ - if ( unlikely(pfn & 0x80000000U) ) return rc; - - ASSERT(m->shadow_dirty_bitmap); - if( likely(pfnshadow_dirty_bitmap_size) ) - { - /* These updates occur with mm.shadow_lock held, so use - (__) version of test_and_set */ - if( __test_and_set_bit( pfn, m->shadow_dirty_bitmap ) == 0 ) - { - // if we set it - m->shadow_dirty_count++; - rc = 1; - } - } - else - { - extern void show_traceX(void); - SH_LOG("mark_dirty OOR! mfn=%x pfn=%x max=%x (mm %p)", - mfn, pfn, m->shadow_dirty_bitmap_size, m ); - SH_LOG("dom=%u caf=%08x taf=%08x\n", - frame_table[mfn].u.domain->domain, - frame_table[mfn].count_and_flags, - frame_table[mfn].type_and_flags ); - } - - return rc; -} - - -static inline int mark_dirty( struct mm_struct *m, unsigned int mfn ) -{ - int rc; - ASSERT(local_irq_is_enabled()); - //if(spin_is_locked(&m->shadow_lock)) printk("+"); - spin_lock(&m->shadow_lock); - rc = __mark_dirty( m, mfn ); - spin_unlock(&m->shadow_lock); - return rc; -} - - -/************************************************************************/ - -static inline void l1pte_write_fault( struct mm_struct *m, - unsigned long *gpte_p, unsigned long *spte_p ) -{ - unsigned long gpte = *gpte_p; - unsigned long spte = *spte_p; - - switch( m->shadow_mode ) - { - case SHM_test: - spte = gpte; - gpte |= _PAGE_DIRTY | _PAGE_ACCESSED; - spte |= _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED; - break; - - case SHM_logdirty: - spte = gpte; - gpte |= _PAGE_DIRTY | _PAGE_ACCESSED; - spte |= _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED; - __mark_dirty( m, (gpte >> PAGE_SHIFT) ); - break; - } - - *gpte_p = gpte; - *spte_p = spte; -} - -static inline void l1pte_read_fault( struct mm_struct *m, - unsigned long *gpte_p, unsigned long *spte_p ) -{ - unsigned long gpte = *gpte_p; - unsigned long spte = *spte_p; - - switch( m->shadow_mode ) - { - case SHM_test: - spte = gpte; - gpte |= _PAGE_ACCESSED; - spte |= _PAGE_ACCESSED; - if ( ! (gpte & _PAGE_DIRTY ) ) - spte &= ~ _PAGE_RW; - break; - - case SHM_logdirty: - spte = gpte; - gpte |= _PAGE_ACCESSED; - spte |= _PAGE_ACCESSED; - spte &= ~ _PAGE_RW; - break; - } - - *gpte_p = gpte; - *spte_p = spte; -} - -static inline void l1pte_no_fault( struct mm_struct *m, - unsigned long *gpte_p, unsigned long *spte_p ) -{ - unsigned long gpte = *gpte_p; - unsigned long spte = *spte_p; - - switch( m->shadow_mode ) - { - case SHM_test: - spte = 0; - if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) == - (_PAGE_PRESENT|_PAGE_ACCESSED) ) - { - spte = gpte; - if ( ! (gpte & _PAGE_DIRTY ) ) - spte &= ~ _PAGE_RW; - } - break; - - case SHM_logdirty: - spte = 0; - if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) == - (_PAGE_PRESENT|_PAGE_ACCESSED) ) - { - spte = gpte; - spte &= ~ _PAGE_RW; - } - - break; - } - - *gpte_p = gpte; - *spte_p = spte; -} - -static inline void l2pde_general( struct mm_struct *m, - unsigned long *gpde_p, unsigned long *spde_p, - unsigned long sl1pfn) -{ - unsigned long gpde = *gpde_p; - unsigned long spde = *spde_p; - - spde = 0; - - if ( sl1pfn ) - { - spde = (gpde & ~PAGE_MASK) | (sl1pfn<shadow_ht[j]; - if(a->pfn){live++; ASSERT(a->spfn_and_flags&PSH_pfn_mask);} - ASSERT((a->pfn&0xf0000000)==0); - ASSERT(a->pfn<0x00100000); - a=a->next; - while(a && live<9999) - { - live++; - if(a->pfn == 0 || a->spfn_and_flags == 0) - { - printk("XXX live=%d pfn=%08lx sp=%08lx next=%p\n", - live, a->pfn, a->spfn_and_flags, a->next); - BUG(); - } - ASSERT(a->pfn); - ASSERT((a->pfn&0xf0000000)==0); - ASSERT(a->pfn<0x00100000); - ASSERT(a->spfn_and_flags&PSH_pfn_mask); - a=a->next; - } - ASSERT(live<9999); - } - - a = m->shadow_ht_free; - while(a) { free++; a=a->next; } - - if(print) printk("Xlive=%d free=%d\n",live,free); - - abs=(perfc_value(shadow_l1_pages)+perfc_value(shadow_l2_pages))-live; - if( abs < -1 || abs > 1 ) - { - printk("live=%d free=%d l1=%d l2=%d\n",live,free, - perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages) ); - BUG(); - } - -} - -#else -#define shadow_audit(p, print) -#endif - - - -static inline struct shadow_status* hash_bucket( struct mm_struct *m, - unsigned int gpfn ) -{ - return &(m->shadow_ht[gpfn % shadow_ht_buckets]); -} - - -static inline unsigned long __shadow_status( struct mm_struct *m, - unsigned int gpfn ) -{ - struct shadow_status **ob, *b, *B = hash_bucket( m, gpfn ); - - b = B; - ob = NULL; - - SH_VVLOG("lookup gpfn=%08x bucket=%p", gpfn, b ); - shadow_audit(m,0); // if in debug mode - - do - { - if ( b->pfn == gpfn ) - { - unsigned long t; - struct shadow_status *x; - - // swap with head - t=B->pfn; B->pfn=b->pfn; b->pfn=t; - t=B->spfn_and_flags; B->spfn_and_flags=b->spfn_and_flags; - b->spfn_and_flags=t; - - if( ob ) - { // pull to front - *ob=b->next; - x=B->next; - B->next=b; - b->next=x; - } - return B->spfn_and_flags; - } -#if SHADOW_HASH_DEBUG - else - { - if(b!=B)ASSERT(b->pfn); - } -#endif - ob=&b->next; - b=b->next; - } - while (b); - - return 0; -} - -/* we can make this locking more fine grained e.g. per shadow page if it -ever becomes a problem, but since we need a spin lock on the hash table -anyway its probably not worth being too clever. */ - -static inline unsigned long get_shadow_status( struct mm_struct *m, - unsigned int gpfn ) -{ - unsigned long res; - - /* If we get here, we know that this domain is running in shadow mode. - We also know that some sort of update has happened to the underlying - page table page: either a PTE has been updated, or the page has - changed type. If we're in log dirty mode, we should set the approrpiate - bit in the dirty bitmap. - NB: the VA update path doesn't use this so needs to be handled - independnetly. - */ - - ASSERT(local_irq_is_enabled()); - //if(spin_is_locked(&m->shadow_lock)) printk("*"); - spin_lock(&m->shadow_lock); - - if( m->shadow_mode == SHM_logdirty ) - __mark_dirty( m, gpfn ); - - res = __shadow_status( m, gpfn ); - if (!res) spin_unlock(&m->shadow_lock); - return res; -} - - -static inline void put_shadow_status( struct mm_struct *m ) -{ - spin_unlock(&m->shadow_lock); -} - - -static inline void delete_shadow_status( struct mm_struct *m, - unsigned int gpfn ) -{ - struct shadow_status *b, *B, **ob; - - ASSERT(spin_is_locked(&m->shadow_lock)); - - B = b = hash_bucket( m, gpfn ); - - SH_VVLOG("delete gpfn=%08x bucket=%p", gpfn, b ); - shadow_audit(m,0); - ASSERT(gpfn); - - if( b->pfn == gpfn ) - { - if (b->next) - { - struct shadow_status *D=b->next; - b->spfn_and_flags = b->next->spfn_and_flags; - b->pfn = b->next->pfn; - - b->next = b->next->next; - D->next = m->shadow_ht_free; - D->pfn = 0; - D->spfn_and_flags = 0; - m->shadow_ht_free = D; - } - else - { - b->pfn = 0; - b->spfn_and_flags = 0; - } - -#if SHADOW_HASH_DEBUG - if( __shadow_status(m,gpfn) ) BUG(); - shadow_audit(m,0); -#endif - return; - } - - ob = &b->next; - b=b->next; - - do - { - if ( b->pfn == gpfn ) - { - b->pfn = 0; - b->spfn_and_flags = 0; - - // b is in the list - *ob=b->next; - b->next = m->shadow_ht_free; - m->shadow_ht_free = b; - -#if SHADOW_HASH_DEBUG - if( __shadow_status(m,gpfn) ) BUG(); -#endif - shadow_audit(m,0); - return; - } - - ob = &b->next; - b=b->next; - } - while (b); - - // if we got here, it wasn't in the list - BUG(); -} - - -static inline void set_shadow_status( struct mm_struct *m, - unsigned int gpfn, unsigned long s ) -{ - struct shadow_status *b, *B, *extra, **fptr; - int i; - - ASSERT(spin_is_locked(&m->shadow_lock)); - - B = b = hash_bucket( m, gpfn ); - - ASSERT(gpfn); - SH_VVLOG("set gpfn=%08x s=%08lx bucket=%p(%p)", gpfn, s, b, b->next ); - - shadow_audit(m,0); - - do - { - if ( b->pfn == gpfn ) - { - b->spfn_and_flags = s; - shadow_audit(m,0); - return; - } - - b=b->next; - } - while (b); - - // if we got here, this is an insert rather than update - - ASSERT( s ); // deletes must have succeeded by here - - if ( B->pfn == 0 ) - { - // we can use this head - ASSERT( B->next == 0 ); - B->pfn = gpfn; - B->spfn_and_flags = s; - shadow_audit(m,0); - return; - } - - if( unlikely(m->shadow_ht_free == NULL) ) - { - SH_LOG("allocate more shadow hashtable blocks"); - - // we need to allocate more space - extra = kmalloc(sizeof(void*) + (shadow_ht_extra_size * - sizeof(struct shadow_status))); - - if( ! extra ) BUG(); // should be more graceful here.... - - memset(extra, 0, sizeof(void*) + (shadow_ht_extra_size * - sizeof(struct shadow_status))); - - m->shadow_extras_count++; - - // add extras to free list - fptr = &m->shadow_ht_free; - for ( i=0; ishadow_ht_extras; - m->shadow_ht_extras = extra; - - } - - // should really put this in B to go right to front - b = m->shadow_ht_free; - m->shadow_ht_free = b->next; - b->spfn_and_flags = s; - b->pfn = gpfn; - b->next = B->next; - B->next = b; - - shadow_audit(m,0); - - return; -} - -static inline void __shadow_mk_pagetable( struct mm_struct *mm ) -{ - unsigned long gpfn, spfn=0; - - gpfn = pagetable_val(mm->pagetable) >> PAGE_SHIFT; - - if ( unlikely((spfn=__shadow_status(mm, gpfn)) == 0 ) ) - { - spfn = shadow_l2_table(mm, gpfn ); - } - mm->shadow_table = mk_pagetable(spfn<pagetable), mm->shadow_mode ); - - if ( unlikely(mm->shadow_mode) ) - { - ASSERT(local_irq_is_enabled()); - spin_lock(&mm->shadow_lock); - - __shadow_mk_pagetable( mm ); - - spin_unlock(&mm->shadow_lock); - } - - SH_VVLOG("leaving shadow_mk_pagetable( gptbase=%08lx, mode=%d ) sh=%08lx", - pagetable_val(mm->pagetable), mm->shadow_mode, - pagetable_val(mm->shadow_table) ); - -} - - -#if SHADOW_DEBUG -extern int check_pagetable(struct mm_struct *m, pagetable_t pt, char *s); -#else -#define check_pagetable(m, pt, s) ((void)0) -#endif - - -#endif /* XEN_SHADOW_H */ - - +#include -- 2.30.2